Librerias

library(tidyverse)
library(plotly)
library(ggthemes)
library(data.table)

Lectura de resultados

breakout_results_rm1m <- fread("breakout_rm1M/results.csv", encoding = "UTF-8") %>% 
  as_tibble()

breakout_results_rm120k <- fread("breakout_rm120k/results.csv", encoding = "UTF-8") %>% 
  as_tibble()

asteroids_results <- fread("asteroids_normal/results.csv", encoding = "UTF-8") %>%
  mutate(
    sum_score = avg_score * n_episodes
  ) %>% as_tibble()

asteroids_c_results <- fread("asteroids_c/results.csv", encoding = "UTF-8") %>% 
  mutate(
    sum_score = avg_score * n_episodes
  ) %>% as_tibble()

asteroids_no_random_results <- fread("asteroids_no_random/results.csv", encoding = "UTF-8") %>% 
  mutate(
    sum_score = avg_score * n_episodes
  ) %>% as_tibble() 

Comparación sobre Breakout

breakout_merge <- full_join(x=breakout_results_rm120k, y=breakout_results_rm1m, by="epoch")
colors <- c("Memoria de 120K" = "darkred", "Memoria de 1M" = "steelblue")

# X = replay memory con 120K
# Y = replay memory con 1M

g <- ggplot(data = breakout_merge %>% dplyr::slice(n = 1:148)) +
  geom_line(mapping = aes(x = epoch, y = avg_q_value.x, color = "Memoria de 120K")) +
  geom_line(mapping = aes(x = epoch, y = avg_q_value.y, color="Memoria de 1M")) +
  labs(x = "Epoch", y = "Valor Q promedio", title = "Valor Q promedio por epoch para Breakout", color = "Tamaño de memoria") +
  scale_color_manual(values = colors)

ggplotly(g)

g <- ggplot(data = breakout_merge %>% dplyr::slice(n = 1:148)) +
  geom_line(mapping = aes(x = epoch, y = avg_score.x, color = "Memoria de 120K")) +
  geom_line(mapping = aes(x = epoch, y = avg_score.y, color="Memoria de 1M")) +
  labs(x = "Epoch", y = "Puntaje promedio", title = "Puntaje promedio por epoch para Breakout", color = "Tamaño de memoria") +
  scale_color_manual(values = colors)

ggplotly(g)

Comparación de aprendizaje entre Asteroids y Breakout

Se usa el método “Local Polynomial Regression Fitting” para hallar las líneas de tendencia.

g <- ggplot(data = asteroids_results) +
  geom_line(mapping = aes(x = epoch, y = avg_score)) +
  labs(x = "Epoch", y = "Puntaje promedio", title = "Puntaje promedio por epoch para Asteroids", color = "") +
  geom_smooth(method="loess", mapping = aes(x = epoch, y = avg_score, color = "Línea de tendencia"), size = 0.5)

ggplotly(g)

g <- ggplot(data = asteroids_results) +
  geom_line(mapping = aes(x = epoch, y = avg_q_value)) +
  labs(x = "Epoch", y = "Valor Q promedio", title = "Valor Q por epoch para Asteroids")

ggplotly(g)

g <- ggplot(data = breakout_results_rm1m) +
  geom_line(mapping = aes(x = epoch, y = avg_score)) +
  labs(x = "Epoch", y = "Puntaje promedio", title = "Puntaje promedio por epoch para Breakout", color = "") +
  geom_smooth(method="loess", mapping = aes(x = epoch, y = avg_score, color = "Línea de tendencia"), size = 0.5)

ggplotly(g)

g <- ggplot(data = breakout_results_rm1m) +
  geom_line(mapping = aes(x = epoch, y = avg_q_value)) +
  labs(x = "Epoch", y = "Valor Q promedio", title = "Valor Q por epoch para Breakout")

ggplotly(g)

Comparacion de los diferentes agentes de Asteroids

asteroids_merge_normal_c <- full_join(x=asteroids_results, y=asteroids_c_results, by="epoch")
colors <- c("Asteroids" = "darkred", "Asteroids con C modificado" = "steelblue", "Línea de tendencia - Asteroids" = "red", "Línea de tendencia - C modificado" = "blue")

# X = replay memory con 120K
# Y = replay memory con 1M

g <- ggplot(data = asteroids_merge_normal_c) +
  geom_line(mapping = aes(x = epoch, y = avg_q_value.x, color = "Asteroids")) +
  geom_line(mapping = aes(x = epoch, y = avg_q_value.y, color="Asteroids con C modificado")) +
  labs(x = "Epoch", y = "Valor Q promedio", title = "Valor Q promedio por epoch para Asteroids", color = "Agente") +
  scale_color_manual(values = colors)

ggplotly(g)

# X = Asteroids normal
# Y = Asteroids con C modificado

g <- ggplot(data = asteroids_merge_normal_c) +
  geom_line(mapping = aes(x = epoch, y = avg_score.x, color = "Asteroids")) +
  geom_line(mapping = aes(x = epoch, y = avg_score.y, color="Asteroids con C modificado")) +
  labs(x = "Epoch", y = "Puntaje promedio", title = "Puntaje promedio por epoch para Asteroids", color = "Agente") +
  geom_smooth(method="loess", mapping = aes(x = epoch, y = avg_score.x, color = "Línea de tendencia - Asteroids")) + 
  geom_smooth(method="loess", mapping = aes(x = epoch, y = avg_score.y, color = "Línea de tendencia - C modificado")) +
  scale_color_manual(values = colors)

ggplotly(g)

g <- ggplot(data = asteroids_results) +
  geom_line(mapping = aes(x = epoch, y = sum_score)) +
  labs(x = "Epoch", y = "Puntaje acumulado", title = "Puntaje acumulado para Asteroids", color = "") +
  geom_smooth(method="loess", mapping = aes(x = epoch, y = sum_score, color = "Línea de tendencia"), size = 0.5)

ggplotly(g)

g <- ggplot(data = asteroids_c_results) +
  geom_line(mapping = aes(x = epoch, y = sum_score)) +
  labs(x = "Epoch", y = "Puntaje acumulado", title = "Puntaje acumulado para Asteroids con C modificado", color = "") +
  geom_smooth(method="loess", mapping = aes(x = epoch, y = sum_score, color = "Línea de tendencia"), size = 0.5)

ggplotly(g)

asteroids_merge_normal_no_random <- full_join(x=asteroids_results, y=asteroids_no_random_results, by="epoch")

colors <- c("Asteroids" = "darkred", "Asteroids sin acciones aleatorias" = "steelblue", "Línea de tendencia - Con aleatoriedad" = "red", "Línea de tendencia - Sin aleatoriedad" = "blue")

# X = replay memory con 120K
# Y = replay memory con 1M

g <- ggplot(data = asteroids_merge_normal_no_random) +
  geom_line(mapping = aes(x = epoch, y = avg_q_value.x, color = "Asteroids")) +
  geom_line(mapping = aes(x = epoch, y = avg_q_value.y, color="Asteroids sin acciones aleatorias")) +
  labs(x = "Epoch", y = "Valor Q promedio", title = "Valor Q promedio por epoch para Asteroids", color = "Agente") +
  scale_color_manual(values = colors)

ggplotly(g)

# X = Asteroids normal
# Y = Asteroids sin acciones aleatorias

g <- ggplot(data = asteroids_merge_normal_no_random) +
  geom_line(mapping = aes(x = epoch, y = avg_score.x, color = "Asteroids")) +
  geom_line(mapping = aes(x = epoch, y = avg_score.y, color = "Asteroids sin acciones aleatorias")) +
  labs(x = "Epoch", y = "Puntaje promedio", title = "Puntaje promedio por epoch para Asteroids", color = "Agente") +
  geom_smooth(method="loess", mapping = aes(x = epoch, y = avg_score.x, color = "Línea de tendencia - Con aleatoriedad")) + 
  geom_smooth(method="loess", mapping = aes(x = epoch, y = avg_score.y, color = "Línea de tendencia - Sin aleatoriedad")) + 
  scale_color_manual(values = colors)

ggplotly(g)

test_scores_random_actions_base <- fread("test_random_actions_asteroids_base.csv", encoding = "UTF-8") %>% 
  as_tibble()

test_scores_random_actions_no_random_actions <- fread("test_random_actions_asteroids_no_random_actions.csv", encoding = "UTF-8") %>% 
  as_tibble()

test_random_actions_base_no_random_actions <- full_join(x=test_scores_random_actions_base, y=test_scores_random_actions_no_random_actions, by="V1")

summary(test_random_actions_base_no_random_actions %>% dplyr::select(-V1))

##  asteroids_base   asteroids_no_random
##  Min.   : 110.0   Min.   : 430       
##  1st Qu.: 580.0   1st Qu.: 980       
##  Median : 930.0   Median :1300       
##  Mean   : 958.6   Mean   :1326       
##  3rd Qu.:1300.0   3rd Qu.:1560       
##  Max.   :3260.0   Max.   :3770

apply(test_random_actions_base_no_random_actions %>% dplyr::select(-V1), 2, sd)

##      asteroids_base asteroids_no_random 
##            446.2923            450.6132

test_random_actions_base_no_random_actions <- test_random_actions_base_no_random_actions %>%
  dplyr::select(-V1) %>%
  melt() %>% 
  mutate(
    variable = case_when(
      variable == "asteroids_base" ~ "Asteroids",
      variable == "asteroids_no_random" ~ "Asteroids sin aleatoriedad",
    )
  ) 

## No id variables; using all as measure variables

g <- ggplot(data = test_random_actions_base_no_random_actions, 
       mapping = aes(x = variable, y = value)) +
  geom_boxplot() +
  labs(x = "Agente", y = "Puntaje", title = "Puntaje de los agentes aleatorios para Asteroids")

ggplotly(g)

Distribuciones t-SNE

tsne_asteroids_normal <- fread("asteroids_normal/tsne_q_values_asteroids_base.csv", encoding = "UTF-8") %>% 
  as_tibble()

g <- ggplot(data = tsne_asteroids_normal, mapping = aes(x = q_values)) +
  geom_density(fill = "lightblue") +
  labs(x = "Valor Q", y = "Densidad", title = "Densidad de los valores Q - Asteroids")

ggplotly(g)

tsne_asteroids_c <- fread("asteroids_c/tsne_q_values_asteroids_c.csv", encoding = "UTF-8") %>% 
  as_tibble()

g <- ggplot(data = tsne_asteroids_c, mapping = aes(x = q_values)) +
  geom_density(fill = "lightblue") +
  labs(x = "Valor Q", y = "Densidad", title = "Densidad de los valores Q - Asteroids con C modificado")

ggplotly(g)

tsne_asteroids_no_random <- fread("asteroids_no_random/tsne_q_values_asteroids_no_random.csv", encoding = "UTF-8") %>% 
  as_tibble()

g <- ggplot(data = tsne_asteroids_no_random, mapping = aes(x = q_values)) +
  geom_density(fill = "lightblue") +
  labs(x = "Valor Q", y = "Densidad", title = "Densidad de los valores Q - Asteroids sin acciones aleatorias")

ggplotly(g)

tsne_breakout_rm1m <- fread("breakout_rm1M/tsne_q_values_breakout_rm1m.csv", encoding = "UTF-8") %>% 
  as_tibble()

g <- ggplot(data = tsne_breakout_rm1m, mapping = aes(x = q_values)) +
  geom_density(fill = "lightblue") +
  labs(x = "Valor Q", y = "Densidad", title = "Densidad de los valores Q - Breakout (Memoria: 1M de entradas)")

ggplotly(g)

tsne_breakout_rm120k <- fread("breakout_rm120K/tsne_q_values_breakout_rm120k.csv", encoding = "UTF-8") %>% 
  as_tibble()

g <- ggplot(data = tsne_breakout_rm120k, mapping = aes(x = q_values)) +
  geom_density(fill = "lightblue") +
  labs(x = "Valor Q", y = "Densidad", title = "Densidad de los valores Q - Breakout (Memoria: 120k entradas)")

ggplotly(g)

comparison_breakout <- full_join(x = tsne_breakout_rm1m, y = tsne_breakout_rm120k, by = "V1")
g <- ggplot(data = comparison_breakout) +
  geom_density(mapping = aes(x = q_values.x, fill = "1.000.000 de entradas") , alpha = 0.5) +
  geom_density(mapping = aes(x = q_values.y, fill = "120.000 entradas"), alpha = 0.5) +
  labs(x = "Valor Q", y = "Densidad", title = "Densidad de los valores Q - Breakout", fill = "Tamaño de la memoria")

ggplotly(g)

comparison_asteroids_base_breakout <- full_join(x = tsne_breakout_rm1m, y = tsne_asteroids_normal, by = "V1")
g <- ggplot(data = comparison_asteroids_base_breakout) +
  geom_density(mapping = aes(x = q_values.x, fill = "Breakout") , alpha = 0.5) +
  geom_density(mapping = aes(x = q_values.y, fill = "Asteroids base"), alpha = 0.5) +
  labs(x = "Valor Q", y = "Densidad", title = "Densidad de los valores Q - Comparación de Breakout y Asteroids", fill = "Agente")

ggplotly(g)

comparison_asteroids_base_c <- full_join(x = tsne_asteroids_c, y = tsne_asteroids_normal, by = "V1")
g <- ggplot(data = comparison_asteroids_base_c) +
  geom_density(mapping = aes(x = q_values.x, fill = "Asteroids con C modificado") , alpha = 0.5) +
  geom_density(mapping = aes(x = q_values.y, fill = "Asteroids base"), alpha = 0.5) +
  labs(x = "Valor Q", y = "Densidad", title = "Densidad de los valores Q - Asteroids", fill = "Agente")

ggplotly(g)

Comparacion de los tres agentes de Asteriods

# Loading data
benchmark_asteroids_normal <- fread("asteroids_normal/benchmark_asteroids_normal.csv", encoding = "UTF-8") %>% 
  as_tibble()
benchmark_asteroids_no_random <- fread("asteroids_no_random/benchmark_asteroids_no_random.csv", encoding = "UTF-8") %>% 
  as_tibble()
benchmark_asteroids_c <- fread("asteroids_c/benchmark_asteroids_c.csv", encoding = "UTF-8") %>% 
  as_tibble()

# Joining results
benchmark <- full_join(full_join(benchmark_asteroids_normal, benchmark_asteroids_no_random, by = "V1"),
                       benchmark_asteroids_c, by = "V1") %>% 
  dplyr::select(-V1)

# Sumarizing information
summary(benchmark)

##    Asteroids      Asteroids - Sin aleatoriedad Asteroids - C modificado
##  Min.   : 110.0   Min.   : 180.0               Min.   : 190.0          
##  1st Qu.: 260.0   1st Qu.: 480.0               1st Qu.: 530.0          
##  Median : 360.0   Median : 630.0               Median : 780.0          
##  Mean   : 396.2   Mean   : 664.8               Mean   : 815.3          
##  3rd Qu.: 480.0   3rd Qu.: 780.0               3rd Qu.:1080.0          
##  Max.   :1800.0   Max.   :1800.0               Max.   :1810.0

apply(benchmark, 2, sd)

##                    Asteroids Asteroids - Sin aleatoriedad 
##                     195.0152                     222.4291 
##     Asteroids - C modificado 
##                     360.2606

benchmark_melt <- benchmark %>% melt()

## No id variables; using all as measure variables

g <- ggplot(data = benchmark_melt, mapping = aes(x = variable, y = value)) +
  geom_boxplot() +
  labs(x = "Agente", y = "Puntaje", title = "Puntaje de los agentes para Asteroids")

ggplotly(g)

Densidad de los valores Q

q_dist_asteroids_normal <- fread("asteroids_normal/q_values_dist_asteroids_normal.csv", encoding = "UTF-8") %>% 
  as_tibble()

g <- ggplot(data = q_dist_asteroids_normal, mapping = aes(x = q_values)) +
  geom_density(fill = "lightblue") +
  labs(x = "Valor Q", y = "Densidad", title = "Densidad de los valores Q - Asteroids")

ggplotly(g)

q_dist_asteroids_c <- fread("asteroids_c/q_values_dist_asteroids_c.csv", encoding = "UTF-8") %>% 
  as_tibble()

g <- ggplot(data = q_dist_asteroids_c, mapping = aes(x = q_values)) +
  geom_density(fill = "lightblue") +
  labs(x = "Valor Q", y = "Densidad", title = "Densidad de los valores Q - Asteroids con C modificado")

ggplotly(g)

q_dist_asteroids_no_random <- fread("asteroids_no_random/q_values_dist_asteroids_no_random.csv", encoding = "UTF-8") %>% 
  as_tibble()

g <- ggplot(data = q_dist_asteroids_no_random, mapping = aes(x = q_values)) +
  geom_density(fill = "lightblue") +
  labs(x = "Valor Q", y = "Densidad", title = "Densidad de los valores Q - Asteroids sin acciones aleatorias")

ggplotly(g)

q_dist_breakout_rm1m <- fread("breakout_rm1M/q_values_dist_breakout_rm1m.csv", encoding = "UTF-8") %>%
  as_tibble()

g <- ggplot(data = q_dist_breakout_rm1m, mapping = aes(x = q_values)) +
  geom_density(fill = "lightblue") +
  labs(x = "Valor Q", y = "Densidad", title = "Densidad de los valores Q - Breakout (Memoria: 1M de entradas)")

ggplotly(g)

q_dist_breakout_rm120k <- fread("breakout_rm120K/q_values_dist_breakout_rm120k.csv", encoding = "UTF-8") %>%
  as_tibble()

g <- ggplot(data = q_dist_breakout_rm120k, mapping = aes(x = q_values)) +
  geom_density(fill = "lightblue") +
  labs(x = "Valor Q", y = "Densidad", title = "Densidad de los valores Q - Breakout (Memoria: 120K entradas)")

ggplotly(g)